package cn.edu.laud.sany.datatest;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.io.IOException;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;

import cn.edu.laud.sany.utils.MyLog;

public class ScanAllDataTest
{
	public static Integer[] cFS;
	public static HashSet<Integer> ignoreParaIds = new HashSet<Integer>();
	public static String csvFolder = "";
	public static String cfsFile = "";
	public static String logFileFolder = "log";
	public static String logFileName = "mylogtest.txt";
	public static boolean dataCleaned = false;
	
	public static long allReadCount = 0;
	public static long allIgnoreCount = 0;
	public static long errorCount = 0;
	
	public static HashMap<Integer, HashMap<String, Long>> allDataCount;
	public static HashSet<String> equipSet;
	
	public static BufferedWriter writer0;
	public static BufferedWriter writer1;
	public static BufferedWriter writer2;
	
	private static MyLog logger;
	
	/**
	 * @param args
	 * @throws ClassNotFoundException
	 * @throws SQLException
	 */
	public static void main(String[] args) throws ClassNotFoundException, SQLException, IOException
	{
		initLaUDMyLog();
		doMain(args);
	}
	
	public static String getLogFilePath()
	{
		return (logFileFolder + "/" + logFileName);
	}
	
	protected static void initLaUDMyLog()
	{
		File logFolderFile = new File(logFileFolder);
		if (logFolderFile.exists() == false)
		{
			logFolderFile.mkdir();
		}
		logger = MyLog.getMyLog(getLogFilePath(), true);
	}
	
	protected static void doMain(String[] args) throws ClassNotFoundException, SQLException,
			IOException
	{
		if (args.length != 3)
		{
			System.err.println("parameter error, usage:");
			System.err.println("\t java -jar scanalldata.jar csvpath cfsfile cleaned");
			return;
		}
		
		parseParams(args);
		
		try
		{
			writer0 = new BufferedWriter(new FileWriter(new File(logFileFolder + "/"
					+ "result0.txt")));
			writer1 = new BufferedWriter(new FileWriter(new File(logFileFolder + "/"
					+ "result1.txt")));
			writer2 = new BufferedWriter(new FileWriter(new File(logFileFolder + "/"
					+ "result2.csv")));
		}
		catch (IOException e2)
		{
			e2.printStackTrace();
		}
		allDataCount = new HashMap<Integer, HashMap<String, Long>>();
		equipSet = new HashSet<String>();
		
		// test csv and cfs files
		if (testCsvFoler() == false)
		{
			System.err.println("can not find csv folder");
			return;
		}
		if (testCfFile() == false)
		{
			System.err.println("can not find CFs file");
			return;
		}
		if (getCFNames() == false)
		{
			System.err.println("can not read CFs correctly");
			return;
		}
		
		getIgnoreParaIds();
		
		long starttime = System.nanoTime();
		logger.info("Scan CSV start, time:" + starttime);
		
		List<String> allFileList = getAllFileList(csvFolder);
		for (String fileName : allFileList)
		{
			File file = new File(fileName);
			if (file.exists() == false)
			{
				logger.info("!File not exist:" + fileName);
				continue;
			}
			BufferedReader bReader = new BufferedReader(new FileReader(file));
			logger.info("read file:" + fileName);
			String lineData = bReader.readLine();
			while ((lineData = bReader.readLine()) != null)
			{
				processData(lineData);
				allReadCount++;
				if (allReadCount % 1000000 == 0)
				{
					logger.info("read count=" + allReadCount + "  find ignore count="
							+ allIgnoreCount + "  error count=" + errorCount);
				}
			}
			logger.info("read count=" + allReadCount + "  find ignore count=" + allIgnoreCount
					+ "  error count=" + errorCount);
			bReader.close();
		}
		long endTime = System.nanoTime();
		logger.info("Scan finish, run time:"
				+ cn.edu.laud.sany.utils.TimeUtils.nanoSecondToString((endTime - starttime))
				+ " read count=" + allReadCount + "  find ignore count=" + allIgnoreCount
				+ "  error count=" + errorCount + "  waiting to write result file.");
		try
		{
			printCountTable();
			writer0.flush();
			writer0.close();
			writer1.flush();
			writer1.close();
			writer2.flush();
			writer2.close();
		}
		catch (IOException e)
		{
			e.printStackTrace();
		}
		endTime = System.nanoTime();
		logger.info("Program's run time:"
				+ cn.edu.laud.sany.utils.TimeUtils.nanoSecondToString((endTime - starttime)));
	}
	
	public static List<String> getAllFileList(String csvFolder)
	{
		List<String> fileProcessMap = new ArrayList<String>();
		File[] files = getCSVFiles(new File(csvFolder));
		for (int i = 0; i < files.length; i++)
		{
			String fileName = files[i].getAbsolutePath();
			if (!fileProcessMap.contains(fileName))
			{
				fileProcessMap.add(fileName);
			}
		}
		return fileProcessMap;
	}
	
	private static File[] getCSVFiles(File file)
	{
		File[] files = new File[0];
		try
		{
			
			files = file.listFiles(new FilenameFilter()
			{
				@Override
				public boolean accept(File dir, String name)
				{
					if (!name.endsWith("csv"))
						return false;
					else
						return true;
				}
			});
		}
		catch (Exception e)
		{
			e.printStackTrace();
		}
		return files;
	}
	
	protected static void printCountTable() throws IOException
	{
		writer2.write("equip,");
		for (int paraId : cFS)
		{
			writer2.write(String.valueOf(paraId));
			writer2.write(",");
		}
		writer2.write("count");
		writer2.write("\r\n");
		for (String equipId : equipSet)
		{
			long count = 0;
			writer2.write(equipId);
			writer2.write(",");
			for (int paraId : cFS)
			{
				long num = 0;
				if (allDataCount.get(paraId).containsKey(equipId))
				{
					num = allDataCount.get(paraId).get(equipId);
					count += num;
					if (num != 0)
					{
						writer1.write(String.valueOf(paraId));
						writer1.write(":");
						writer1.write(equipId);
						writer1.write("=");
						writer1.write(String.valueOf(num));
						writer1.write("\r\n");
						writer1.flush();
					}
				}
				writer2.write(String.valueOf(num));
				writer2.write(",");
			}
			
			writer2.flush();
			writer2.write(String.valueOf(count));
			writer2.write("\r\n");
			
		}
		
		for (int paraId : cFS)
		{
			long count = 0;
			Iterator<Long> iter = allDataCount.get(paraId).values().iterator();
			while (iter.hasNext())
			{
				long num = iter.next();
				count += num;
			}
			writer0.write(String.valueOf(paraId));
			writer0.write("=");
			writer0.write(String.valueOf(count));
			writer0.write("\r\n");
			writer0.flush();
		}
	}
	
	protected static void processData(String string)
	{
		String data[] = null;
		try
		{
			data = string.split(",");
			int paraId;
			String equipCode;
			if (dataCleaned)
			{
				if (data.length != 4)
				{
					logger.error("data format error.(org:)" + string);
					allIgnoreCount++;
					errorCount++;
					return;
				}
				paraId = Integer.valueOf(data[1]);
				equipCode = data[0];
			}
			else
			{
				if (data.length != 6)
				{
					logger.error("data format error.(org:)" + string);
					allIgnoreCount++;
					errorCount++;
					return;
				}
				paraId = Integer.valueOf(trimData(data[2]));
				equipCode = trimData(data[1]);
			}
			if (ignoreParaIds.contains(paraId))
			{
				allIgnoreCount++;
				return;
			}
			if (!equipSet.contains(equipCode))
			{
				equipSet.add(equipCode);
			}
			HashMap<String, Long> nowDataCount = allDataCount.get(paraId);
			long count = 0;
			if (nowDataCount.containsKey(equipCode))
			{
				count = nowDataCount.get(equipCode);
			}
			count++;
			nowDataCount.put(equipCode, count);
		}
		catch (NumberFormatException e1)
		{
			e1.printStackTrace();
			logger.error(e1.getMessage() + "(ori:)" + string);
			allIgnoreCount++;
			errorCount++;
			return;
		}
	}
	
	private static String trimData(String oriData)
	{
		oriData = oriData.trim();
		oriData = oriData.substring(1, oriData.length() - 1);
		return oriData;
	}
	
	protected static void parseParams(String[] args)
	{
		csvFolder = args[0];
		cfsFile = args[1];
		if (args[2].equals("1"))
		{
			dataCleaned = true;
		}
		logger.info("csv path = " + csvFolder);
		logger.info("cfs file = " + cfsFile);
		logger.info("data cleaned = " + dataCleaned);
	}
	
	protected static boolean getCFNames()
	{
		boolean flag = true;
		File file = new File(cfsFile);
		BufferedReader bReader = null;
		String paraIdString = null;
		ArrayList<Integer> cfList = new ArrayList<Integer>();
		try
		{
			bReader = new BufferedReader(new FileReader(file));
			while ((paraIdString = bReader.readLine()) != null)
			{
				paraIdString = paraIdString.trim();
				if (paraIdString.equals(""))
				{
					continue;
				}
				int paraId = Integer.valueOf(paraIdString);
				cfList.add(paraId);
				allDataCount.put(paraId, new HashMap<String, Long>());
			}
			
		}
		catch (IOException e1)
		{
			flag = false;
			logger.error("can not open cf file");
		}
		catch (NumberFormatException e2)
		{
			flag = false;
			logger.error("cf file contains invalid integer");
		}
		finally
		{
			if (flag == false)
			{
				logger.info("get cfs failed!");
			}
			try
			{
				if (bReader != null)
				{
					bReader.close();
				}
			}
			catch (IOException e)
			{
				logger.info(e.getMessage());
			}
		}
		if (flag == true)
		{
			cFS = cfList.toArray(new Integer[0]);
		}
		return flag;
	}
	
	protected static boolean testCsvFoler()
	{
		try
		{
			File file = new File(csvFolder);
			if (file.exists())
			{
				return true;
			}
		}
		catch (Exception e)
		{
			e.printStackTrace();
		}
		return false;
	}
	
	protected static boolean testCfFile()
	{
		try
		{
			File file = new File(cfsFile);
			if (file.exists())
			{
				return true;
			}
		}
		catch (Exception e)
		{
			e.printStackTrace();
		}
		return false;
	}
	
	protected static boolean getIgnoreParaIds()
	{
		ignoreParaIds = new HashSet<Integer>();
		ignoreParaIds.add(356);
		return true;
	}
	
}
